#!/usr/bin/env python3
#-*- coding:utf-8 -*-

import os
import sys
import pandas as pd
import numpy as np
from pyquery import PyQuery as pq
import random

import conf

import diff_rank

def send_mail(mail_content):
    # Import smtplib for the actual sending function
     import smtplib

     # Import the email modules we'll need
     from email.mime.text import MIMEText
     from email.header import Header

     # Open a plain text file for reading.  For this example, assume that
     # the text file contains only ASCII characters.
     # Create a text/plain message

     msg = MIMEText(mail_content, _subtype='html', _charset='utf-8')
     # me == the sender's email address
     # you == the recipient's email address
     msg['Subject'] = Header("平搜diff指标daily report", "utf-8")
     msg['From'] = "zhangyansheng@jd.com"
     #msg['To'] = "zhangyansheng@jd.com;chenjianyu@jd.com;wenke@jd.com;yangjun6@jd.com;"
     #msg['Cc'] = "wangchao29@jd.com; wangchunming@jd.com;liyouke@jd.com;"
     msg['To'] = "zhangyansheng@jd.com;"

     # Send the message via our own SMTP server.
     s = smtplib.SMTP('smtp.jd.local')
     s.send_message(msg)
     s.quit()
 
     return

"""
对top torso tail query每天进行diff，并以邮件发送关注人
"""

def merge_diff_result(query_file):
    """
    根军query_file的diff结果，将各个情况下的top5 的diff结果merger在一起并返回
    """
    data = pd.read_table(query_file, names = ['query'])
    data = data.set_index("query")

    file_base_name = os.path.basename(query_file)

    for sort_type in conf.sort_type:
        diff_result_file = "{}/{}_{}.diff".format(conf.output_path,file_base_name,sort_type)
        diff_data = pd.read_table(diff_result_file, names =
            ['query', 'top5', 'top10', 'top20', 'top30', 'top60', 'top180', 'top300', 'top600'])
        # 只关注top5的结果，其它的全部丢弃
        diff_data = diff_data.set_index("query")['top5']
        # 通过DataFrame选出的是Series，可以直接设置name
        diff_data.name = sort_type
        data = data.join(diff_data, how = "inner")

    return data

def get_less0dot2_count(diff_data):
    less2dot2_count = pd.Series()

    for sort_type in conf.sort_type:
        tmp_data = diff_data[diff_data[sort_type] <= 0.2]
        less2dot2_count[sort_type] = len(tmp_data)

    return less2dot2_count / len(diff_data)

def get_sample_query(diff_data):
    sample_query = pd.Series()

    for sort_type in conf.sort_type:
        tmp_data = diff_data[diff_data[sort_type] == 0]
        sample_query[sort_type] = " | ".join(tmp_data.index[:5])

    return sample_query

def get_report_data(query_file):
    query_origin_file = query_file + "_origin"

    query_origin_data = pd.read_table(query_origin_file, names =
        ['query', 'cid1', 'cid1_name', 'pv'])

    if len(query_origin_data) == 0:
        print("query_origin_data empty")
        return

    query_origin_data = query_origin_data.set_index("query")

    diff_data = merge_diff_result(query_file)
    if len(diff_data) == 0:
        print("merge_diff_result empty")
        return

    diff_data = diff_data.join(query_origin_data, how = "left")

    cid1s = query_origin_data.set_index(['cid1','cid1_name']).index.unique()

    diff = pd.DataFrame()
    diff["overall"] = diff_data.mean()

    less0dot2 = pd.DataFrame()
    less0dot2['overall'] = get_less0dot2_count(diff_data)

    sample_query = pd.DataFrame()
    sample_query['overall'] = get_sample_query(diff_data)

    for cid1,cid1_name in cid1s:
        tmp_data = diff_data[diff_data['cid1'] == cid1]
        cid1_key = "{}({})".format(cid1_name,cid1)
        diff[cid1_key] = tmp_data.mean()
        less0dot2[cid1_key] = get_less0dot2_count(tmp_data)
        sample_query[cid1_key] = get_sample_query(tmp_data)

    return diff,less0dot2,sample_query

def populate_table(table,row_template,data):

    for key in data.index:
        row = row_template.clone()
        row(".Cid3").html(str(key))

        val = data.ix[key]

        for sort_type in conf.sort_type:
            row(".{}".format(sort_type)).html(str(val[sort_type]))

        table.append(row)

    return table

def craw_diff(platform, query_files):
    if platform == "PC":
        conf.url1 = "http://pc.p-search.jd.local/status?pagesize=60&debug=yes&qp_disable=no&area_ids=1,72,2799&use_blender_cache=no&usecache=no&use_detail_server=yes&use_pagecache=no&"
        conf.url2 = "http://pc.search.jd.local/status?pagesize=60&debug=yes&qp_disable=no&area_ids=1,72,2799&use_blender_cache=no&usecache=no&use_detail_server=yes&use_pagecache=no&"
    elif platform == "APP":
        conf.url1 = "http://mobile.p-search.jd.local/status?pagesize=60&debug=yes&qp_disable=no&area_ids=1,72,2799&use_blender_cache=no&usecache=no&use_detail_server=yes&use_pagecache=no&"
        conf.url2 = "http://mobile.search.jd.local/status?pagesize=60&debug=yes&qp_disable=no&area_ids=1,72,2799&use_blender_cache=no&usecache=no&use_detail_server=yes&use_pagecache=no&"
    else:
        return False


    conf.sort_type = ['sort_default',
              'sort_dredisprice_desc',
              'sort_dredisprice_asc',
              'sort_commentcount_desc',
              'sort_totalsales15_desc',
              'sort_winsdate_desc']

    for query_file in query_files:
        print("crawl {} {} ...".format(platform,query_file))
        diff_rank.crawl_query_file(query_file)

    return True

def main():
    query_files = [
            'query/20161008_top_10000',
            'query/20161008_torso_10000',
            'query/20161008_tail_10000',
            ]

    """
    query_files = [
            'query/20161008_top_3000',
            'query/20161008_torso_3000',
            'query/20161008_tail_3000',
            ]

    query_files = [
            'query/20161008_top_100',
            'query/20161008_torso_100',
            'query/20161008_tail_100',
            ]
    """

    platforms = ['PC','APP']

    template_file = "report_template.html"
    template = pq(open(template_file, encoding="utf-8").read())

    main_content = template("#MainContent")
    main_content_template = template("#MainContentTemplate")
    diff_info_template = template("#DiffInfoTemplate")
    table_tempalte = template("#TableTemplate")
    table_row_template = template("#TableRow")

    b_success = False

    for i,platform in enumerate(platforms):
        i += 1

        print("craw diff {} ".format(platform))

        if not craw_diff(platform, query_files):
            continue

        print("generate {} report".format(platform))

        diff_title = "{0}. 平搜{1}集群 vs HD {1}集群相同率统计结果".format(i,platform)

        tmp_content = main_content_template.clone()
        tmp_content("#DiffTitle").html(diff_title)

        diff_content = tmp_content(".Content")

        b_success = True

        for j,query_file in enumerate(query_files):
            j += 1
            file_base_name = os.path.basename(query_file)
            items = file_base_name.split("_")

            if len(items) != 3:
                print("file path error", query_file)
                continue

            date,query_type,count = items

            tmp_diff_info_template = diff_info_template.clone()

            tmp_diff_info_template("#QueryTypeTitle").html(
                    "{}.{}. {} {} 相同率平均值：".format(i,j,query_type, count))
            tmp_diff_info_template("#Less0Dot2Title").html(
                    "{}.{}.1 {} {} 相同率小于0.2的query占比：".format(i,j, query_type, count))
            tmp_diff_info_template("#SampleQueryTitle").html(
                    "{}.{}.2 {} {} 相同率为0的query sample：".format(i,j, query_type, count))

            diff,less0dot2,sample_query = get_report_data(query_file)

            tab = populate_table(table_tempalte.clone()("table"), table_row_template, diff.T)
            tmp_diff_info_template("#AvgDiffTable").append(tab)

            tab = populate_table(table_tempalte.clone()("table"), table_row_template, less0dot2.T)
            tmp_diff_info_template("#Less0Dot2Table").append(tab)

            tab = populate_table(table_tempalte.clone()("table"), table_row_template, sample_query.T)
            tmp_diff_info_template("#SampleQueryTable").append(tab)

            diff_content.append(tmp_diff_info_template)

        main_content.append(tmp_content)

    # 清空template，使之不显示
    main_content_template.html("")
    table_tempalte.html("")
    diff_info_template.html("")
    table_row_template.html("")

    if b_success:
        print("sending mail...")
        content = str(template)

        with open("result.html","w") as f:
            f.write(content)

        send_mail(content)
    else:
        print("some thing to be wrong")

if __name__=='__main__':
    main()
    #send_mail(open("result.html",encoding = "utf-8").read())
    print("done")

